package cn.buaa.HLJ.Crawler;

import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.http.HttpHost;
import org.apache.ibatis.io.Resources;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.session.SqlSessionFactoryBuilder;

import cn.buaa.HLJ.Entity.PapersDetail;
import cn.buaa.HLJ.Entity.ProjectUrl;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class SpringerCrawler  implements  PageProcessor,Runnable  {
	private Site site;
	private SqlSession session;
	
	private boolean add=false;
	private int begin=0;
	private boolean keep=true;
	private List<ProjectUrl> urls=new ArrayList<ProjectUrl>();
	
	
	public final static String separator = " ## ";
	public final static String separator2 = " $$ ";
	
	
	private List<String>  excepurls=new ArrayList<String>();
	
	private int  exceptionurlnum=0;
	
	public SpringerCrawler() throws IOException
	{
		String resource="mybatis_config.xml";
		Reader reader=Resources.getResourceAsReader(resource);
		SqlSessionFactory sessionFactory=new SqlSessionFactoryBuilder().build(reader);
		session=sessionFactory.openSession();	
		
		//excepurls=Excep.ExcepUrls("ExceptionUrls.txt");
		String  urlstate="cn.buaa.HLJ.Mapper.ProjectUrlMapper.getDblpChinese";
		
	    urls=session.selectList(urlstate); // 
	    
	    /*String statement="cn.buaa.HLJ.Mapper.PapersDetailMapper.getPapersDetailbyid";
	    PapersDetail detail = new PapersDetail();
	    List<PapersDetail> urls1=session.selectList(statement);
	    
	    String statement2="cn.buaa.HLJ.Mapper.PapersDetailMapper.insertPapersDetail";
	    detail.setTitle("sssss");
	    session.insert(statement2, detail);
	    session.commit();
	    
	    System.out.println(urls1.size());
	    System.out.println(urls1.get(0).getTitle());*/
	    
	    /*for(ProjectUrl cur : urls){
	    	System.out.println(cur.getEe());
	    }*/
	}

	@Override
	public void run() {
		
		//String s="http://d.wanfangdata.com.cn/Free/Cstad/0400320071";
		
		//String s=urls.get(begin).getUrl();
		//String s = "http://link.springer.com/article/10.1007%2Fs00779-004-0300-0";
		
		System.out.println("begin = "+begin);
		
		//String s=excepurls.get(begin);
		
		List<String> url = new ArrayList<String>();
		url.clear();
		// url.add(s.replaceAll("\"", "%22"));
		
		for(ProjectUrl p : urls){
			if(p.getEe().contains("http://dx.doi.org/10.1007")){
				//System.out.println(p.getEe());
				url.add(p.getEe().replaceAll("\"", "%22"));
			}
		}
		
		System.out.println("url.size() = "+url.size());
		
		//System.out.println(s.replaceAll("\"", "%22"));
		
		Spider.create(this).startUrls(url)
		.thread(2)
		.run(); 
		
	}

	@Override
	public Site getSite() {
		//site = Site.me().setSleepTime(1000).setRetryTimes(5).setCycleRetryTimes(3).setTimeOut(60000).setHttpProxy(new HttpHost("112.25.41.136",80));
		site=Site.me().setSleepTime(1000).setRetryTimes(5).setCycleRetryTimes(3).setTimeOut(60000);
		site.setCharset("UTF-8");
		return site;
	}

	@Override
	public void process(Page page) {
		keep =true;
		
		PapersDetail detail = new PapersDetail();
		
		String someDetails = "";
		String serialNumber = "";
		
		
		Map<Integer, String> schools = new HashMap<Integer, String>(); // 锟斤拷录锟斤拷前页锟斤拷锟斤拷值锟窖ｏ拷锟斤拷锟斤拷诺锟斤拷锟斤拷
		
		List<String> marks = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dt/text()").all();
		List<String> code = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd/text()").all();
		
		String theAbstract = page.getHtml().xpath("//div/section[@class='Abstract']/p[@class='Para']/allText()").toString();
		System.out.println("  " + "Abstract"+": "+theAbstract);
		
		detail.setBrief(theAbstract);
		
		if(code.size()==marks.size()){
			
			System.out.println("code.size(): "+code.size());
			
			for(int i=0;i<	code.size();i++){
				
				
				if(marks.get(i).equals("Title")){ // Mining Biomedical Text towards Building a Quantitative Food-Disease-Gene Network
					System.out.println("  " + marks.get(i)+": "+code.get(i));
					detail.setTitle(code.get(i));
				}
				
				else if(marks.get(i).equals("Book Title")){ // Learning Structure and Schemas from Documents
					String bookTitle = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd[@id='abstract-about-publication']//a/text()").toString();
					System.out.println("  " + marks.get(i)+": "+bookTitle);
					detail.setBookTitle(bookTitle);
				}
				
				else if(marks.get(i).equals("Journal")){ // Computational Intelligence ## Artificial Intelligence (incl. Robotics)
					
					String journalName = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd/a[@id='abstract-about-publication']/text()").toString();
					String journalDetail = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd/a[@id='abstract-about-issue']/text()").toString();
					
					String result = journalName + separator + journalDetail;
					System.out.println("  " + marks.get(i)+": "+ result);
					detail.setJournal(result);
				}
				
				
				else if(marks.get(i).equals("Cover Date")){ // 2011-02
					//System.out.println("  " + marks.get(i)+": "+ code.get(i));
					someDetails += marks.get(i)+":"+ code.get(i) + separator;
				}
				else if(marks.get(i).equals("Pages")){ // pp 205-225
					//System.out.println("  " + marks.get(i)+": "+ code.get(i));
					someDetails += marks.get(i)+":"+ code.get(i) + separator;
				}
				else if(marks.get(i).equals("Copyright")){ // 2011
					//System.out.println("  " + marks.get(i)+": "+ code.get(i));
					someDetails += marks.get(i)+":"+ code.get(i) + separator;
				}
				
				
				else if(marks.get(i).equals("DOI")){ // 10.1007/978-3-642-22913-8_10
					System.out.println("  " + marks.get(i)+": "+ code.get(i));
					detail.setDoi(code.get(i));
				}
				
				
				
				
				else if(marks.get(i).equals("Print ISBN")){ // 978-3-642-22912-1
					//System.out.println("  " + marks.get(i)+": "+ code.get(i));
					serialNumber += marks.get(i)+":"+ code.get(i) + separator;
				}
				else if(marks.get(i).equals("Online ISBN")){ // 978-3-642-22913-8
					//System.out.println("  " + marks.get(i)+": "+ code.get(i));
					serialNumber += marks.get(i)+":"+ code.get(i) + separator;
				}
				else if(marks.get(i).equals("Series ISSN")){ // 1860-949X
					//System.out.println("  " + marks.get(i)+": "+ code.get(i));
					serialNumber += marks.get(i)+":"+ code.get(i) + separator;
				}
				else if(marks.get(i).equals("Print ISSN")){ // 0001-5903
					//System.out.println("  " + marks.get(i)+": "+ code.get(i));
					serialNumber += marks.get(i)+":"+ code.get(i) + separator;
				}
				else if(marks.get(i).equals("Online ISSN")){ // 1432-0525
					//System.out.println("  " + marks.get(i)+": "+ code.get(i));
					serialNumber += marks.get(i)+":"+ code.get(i) + separator;
				}
				
				
				
				
				else if(marks.get(i).equals("Series Title")){ // Studies in Computational Intelligence
					String seriesTitle = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd[@id='abstract-about-book-series-title']//a/text()").toString();
					System.out.println("  " + marks.get(i)+": "+seriesTitle);
					detail.setSeriesTitle(seriesTitle);
				}
				else if(marks.get(i).equals("Series Volume")){ // 375
					//System.out.println("  " + marks.get(i)+": "+ code.get(i));
					someDetails += marks.get(i)+":"+ code.get(i) + separator;
				}
				
				else if(marks.get(i).equals("Publisher")){ // Springer Berlin Heidelberg
					System.out.println("  " + marks.get(i)+": "+ code.get(i));
					detail.setPublisher(code.get(i));
				}
				else if(marks.get(i).equals("Copyright Holder")){ // Springer-Verlag GmbH Berlin Heidelberg
					System.out.println("  " + marks.get(i)+": "+ code.get(i));
					detail.setCopyrightHolder(code.get(i));
				}
				
				else if(marks.get(i).equals("Topics")){ // Computational Intelligence ## Artificial Intelligence (incl. Robotics)
					
					List<String> topics = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd[@itemprop='genre']/ul[@class='abstract-about-subject']//a/text()").all();
					
					String result = "";
					for(int j=0;j < topics.size();j++){
						if(j > 0){
							result += separator;
						}
						result += topics.get(j);
					}
					System.out.println("  " + marks.get(i)+": "+ result);
					detail.setTopics(result);
				}
				
				else if(marks.get(i).equals("Keywords")){ // Tangible user interface ## Storytelling
					List<String> sectors = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd[@itemprop='keywords']/ul[@class='abstract-keywords']/li/text()").all();
					
					String result = "";
					for(int j=0;j < sectors.size();j++){
						if(j > 0){
							result += separator;
						}
						result += sectors.get(j);
					}
					System.out.println("  " + marks.get(i)+": "+ result);
					detail.setKeywords(result);
				}
				
				else if(marks.get(i).equals("Industry Sectors")){ // Electronics ## Telecommunications ## IT & Software
					List<String> sectors = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd[@itemprop='genre']/ul[@class='abstract-about-industrysectors']//a/text()").all();
					
					String result = "";
					for(int j=0;j < sectors.size();j++){
						if(j > 0){
							result += separator;
						}
						result += sectors.get(j);
					}
					System.out.println("  " + marks.get(i)+": "+ result);
					detail.setIndustrySectors(result);
				}
					
				
				
				
				
				else if(marks.get(i).equals("Editor Affiliations")){
					List<String> eaNum = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd/ul[@class='editor-affiliations']//span[@class='position']/text()").all();
					List<String> ea = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd/ul[@class='editor-affiliations']//span[@class='affiliation']/text()").all();
				
					for(int j=0;j<eaNum.size();j++){
						//System.out.println("sss    "+eaNum.get(j).replaceAll("[^\\d]+", "")+"   "+ea.get(j));
						
						int eaId = Integer.parseInt(eaNum.get(j).replaceAll("[^\\d]+", ""));
						
						if(schools.get(eaId) == null){
							schools.put(eaId, ea.get(j));
						}
					}
				}
				
				else if(marks.get(i).equals("Author Affiliations")){
					List<String> aaNum = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd/ul[@class='author-affiliations']//span[@class='position']/text()").all();
					List<String> aa = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd/ul[@class='author-affiliations']//span[@class='affiliation']/text()").all();
				
					for(int j=0;j<aaNum.size();j++){
						//System.out.println("sss    "+eaNum.get(j).replaceAll("[^\\d]+", "")+"   "+ea.get(j));	
						int eaId = Integer.parseInt(aaNum.get(j).replaceAll("[^\\d]+", ""));
						
						if(schools.get(eaId) == null){
							schools.put(eaId, aa.get(j));
						}
					}
				}
				
				else if(marks.get(i).equals("Authors")){ // Hui Yang $$ (4) ## Rajesh Swaminathan $$ (4) ## Abhishek Sharma $$ (4) ## Vilas Ketkar $$ (4) ## Jason D锟斤拷Silva $$ (4)
					List<String> authors = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd/ul[@class='authors']//a[@class='person']/text()").all();
					List<String> authorsOrgNum = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd/ul[@class='authors']//sup/text()").all();
					
					String result = "";
					for(int j=0;j<authors.size();j++){
						if(j > 0){
							result += separator;
						}
						result += authors.get(j) + separator2 + authorsOrgNum.get(j);
					}
					System.out.println("  " + marks.get(i)+": "+ result);
					detail.setAuthors(result);
				}
				
				else if(marks.get(i).equals("Editors")){ // Marenglen Biba $$ (2) ## Fatos Xhafa $$ (3)
					List<String> editors = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd/ul[@class='editors']//a[@class='person']/text()").all();
					List<String> editorsOrgNum = page.getHtml().xpath("//div[@class='expander-content']//div[@class='summary']/dl/dd/ul[@class='editors']//sup/text()").all();
					
					String result = "";
					for(int j=0;j<editors.size();j++){
						if(j > 0){
							result += separator;
						}
						result += editors.get(j) + separator2 + editorsOrgNum.get(j);
					}
					System.out.println("  " + marks.get(i)+": "+ result);	
					detail.setEditors(result);
				}
				
				
				else{
					System.out.println("  漏网之鱼 " + marks.get(i)+":   "+code.get(i));
				}
				
				
			}
			
			//2 $$  University of New York Tirana  ## 3 $$  Technical University of Catalonia  ## 4 $$  Department of Computer Science, San Francisco State University, USA 
			String schoolResult = "";
			boolean isFirstSchool = true;
			for (Map.Entry<Integer, String> entry : schools.entrySet()) {
				if(!isFirstSchool){
					schoolResult += separator;	
				}
				isFirstSchool = false;
				schoolResult += entry.getKey() + separator2 + entry.getValue();
			}
			System.out.println("  " + "schools"+": "+ schoolResult);
			detail.setSchools(schoolResult);
			
			System.out.println("  " + "serialNumber"+": "+ serialNumber);
			detail.setSerialNumber(serialNumber);
			
			System.out.println("  " + "someDetails"+": "+ someDetails);
			detail.setSomeDetails(someDetails);
		}
		


		
		detail.setUrl(page.getUrl().toString());
		
		
		
		
		System.out.println("****************************************************");
		
		String statement="cn.buaa.HLJ.Mapper.PapersDetailMapper.insertPapersDetail";
		
		//detail = new PapersDetail();
		//detail.setTitle("aaaafsggf");
		System.out.println(detail.getTitle());
		
		session.insert(statement, detail);
		session.commit();
		
		/*
		
		if(code.size()!=marks.size())
		{
			Excep.PrintURL("Exception2.txt",page.getUrl().toString());
			exceptionurlnum++;
			keep=false;
		}
		
		
		if(keep)
		{
			//System.out.println("afasfea");
			session.insert(statement, detail);
			session.commit();
		}
		
		
		
		
		
		if(add==false)
		{
			
			List<String> tempurls = new ArrayList<String>();
			for(ProjectUrl match:urls)
			{
				tempurls.add(match.getUrl());
			}
			page.addTargetRequests(tempurls);
			
			//page.addTargetRequests(excepurls);
			
			add=true;
		}*/
		
	}
	
	public static void main(String[] args) throws IOException{
		SpringerCrawler project=new SpringerCrawler();
		project.run();
		System.out.println("exceptionurlnum = "+project.exceptionurlnum);
		System.out.println("pass");
	}
}
